/************************************************************
 * DS_TOKENIZER.HPP                                10.11.2006
 *
 * .
 ************************************************************/

/*************************************************************
 * FindDelim()
 * 
 *************************************************************/
inline int FindDelim(const dsSTRING & str, int start_pos, const dsSTRING & delims)
{
   int min_pos = 0xFFFF;
   for(int i = 0; i != delims.Length(); i++) {
      int pos = str.Find(delims[i], start_pos);
      if (pos != -1) {
         if (pos < min_pos) { 
            min_pos = pos;
         }
      }
   }

   if(min_pos != 0xFFFF) {
      return min_pos;
   }

   return -1;
}

/*************************************************************
* Tokenize()
* 
*************************************************************/
template <class PROCESSOR_T>
TOKENIZE_RESULT Tokenize(dsSTRING const& str, dsSTRING const& delims, PROCESSOR_T & processor)
{
   TOKENIZE_RESULT pr = TOKENIZE_CONTINUE;
   bool done = false;
   int base_pos = 0;
   int next_pos = FindDelim(str, 0, delims);

   dsVECTOR<char> token;
   token.Reserve(32);

   while(!done) {
      int length = 0;
      if(next_pos == -1) {
         length = str.Length() - base_pos;
         done = true;
      } else {
         length = next_pos - base_pos;
      }
      
      token.Resize(length + 1);
      memcpy(token.Begin(), str.CStr() + base_pos, length);
      token[length] = 0;

		if (!done || length != 0) {
			pr = processor.ProcessToken(token.Begin(), length);
		}
      if (pr != TOKENIZE_CONTINUE) {
         break;
      }
      if (next_pos != -1) {
         pr = processor.ProcessDelim(str[next_pos]);
         if (pr != TOKENIZE_CONTINUE) {
            break;
         }
      }
      base_pos = next_pos + 1;
      next_pos = FindDelim(str, base_pos, delims);
   }
   return pr;
}

//
// End-of-file DS_TOKENIZER.HPP
//
